{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copyright 2020 Google LLC\n",
    "#\n",
    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "# you may not use this file except in compliance with the License.\n",
    "# You may obtain a copy of the License at\n",
    "#\n",
    "#     https://www.apache.org/licenses/LICENSE-2.0\n",
    "#\n",
    "# Unless required by applicable law or agreed to in writing, software\n",
    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "# See the License for the specific language governing permissions and\n",
    "# limitations under the License."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a target=\"_blank\" href=\"https://colab.research.google.com/github/GoogleCloudPlatform/keras-idiomatic-programmer/blob/master/books/deep-learning-design-patterns/Workshops/Junior/Deep%20Learning%20Design%20Patterns%20-%20Workshop%20-%20Chapter%204.ipynb\">\n",
    "<img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deep Learning Design Patterns - Code Labs\n",
    "\n",
    "## Lab Exercise #8 - Get Familiar with Mobile Convolutional Networks\n",
    "\n",
    "## Prerequistes:\n",
    "\n",
    "    1. Familiar with Python\n",
    "    2. Completed Chapter 4: Mobile Convolutional Networks\n",
    "\n",
    "## Objectives:\n",
    "\n",
    "    1. Use metaparameter to thin a MobileNet v1.\n",
    "    2. Code a mobile convolutional network style classifier.\n",
    "    3. Code a SqueezeNet fire block\n",
    "    4. Quantize a mobile convolutional network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Installs\n",
    "\n",
    "For the last task, we will use opencv. So let's install it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting opencv-python\n",
      "  Downloading opencv_python-4.2.0.34-cp37-cp37m-manylinux1_x86_64.whl (28.2 MB)\n",
      "\u001b[K     |████████████████████████████████| 28.2 MB 4.9 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.14.5 in /opt/conda/lib/python3.7/site-packages (from opencv-python) (1.18.1)\n",
      "Installing collected packages: opencv-python\n",
      "Successfully installed opencv-python-4.2.0.34\n"
     ]
    }
   ],
   "source": [
    "!pip install opencv-python"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Use Metaparameter to Thin a MobileNet v1\n",
    "\n",
    "Let's start with a MobileNet v1 coded using the procedural reuse design pattern.\n",
    "\n",
    "\n",
    "You will need to:\n",
    "\n",
    "    1. Set the thinning factor (width multiplier) are various locations in the code.\n",
    "    2. Set the max value for ReLU to clip values above the max value.\n",
    "    3. Calculate the number of thinned filters in the mobilenet blocks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import Input, Model\n",
    "from tensorflow.keras.layers import ZeroPadding2D, Conv2D, BatchNormalization, ReLU\n",
    "from tensorflow.keras.layers import DepthwiseConv2D, GlobalAveragePooling2D, Reshape, Dropout\n",
    "\n",
    "\n",
    "def stem(inputs, alpha):\n",
    "    \"\"\" Construct the Stem Group\n",
    "        inputs : input tensor\n",
    "        alpha  : width multiplier\n",
    "    \"\"\"\n",
    "    # Convolutional block\n",
    "    # Replace the ?? by the thinning factor\n",
    "    # HINT: reduce the number of filters (32) by the thinning factor\n",
    "    x = ZeroPadding2D(padding=((0, 1), (0, 1)))(inputs)\n",
    "    x = Conv2D(32 * ??, (3, 3), strides=(2, 2), padding='valid')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    # Replace the max value to clip\n",
    "    # HINT: the best value found by the authors\n",
    "    x = ReLU(??)(x)\n",
    "\n",
    "    # Depthwise Separable Convolution Block\n",
    "    x = depthwise_block(x, 64, alpha, (1, 1))\n",
    "    return x\n",
    " \n",
    "# Relace the ?? by the parameter for thinning\n",
    "# HINT: the same parameter name as in the stem()\n",
    "def learner(x, ??):\n",
    "    \"\"\" Construct the Learner\n",
    "        x      : input to the learner\n",
    "        alpha  : width multiplier\n",
    "    \"\"\"\n",
    "    # First Depthwise Separable Convolution Group\n",
    "    x = group(x, 128, 2, alpha)\n",
    "\n",
    "    # Second Depthwise Separable Convolution Group\n",
    "    x = group(x, 256, 2, alpha)\n",
    "\n",
    "    # Third Depthwise Separable Convolution Group\n",
    "    x = group(x, 512, 6, alpha)\n",
    "\n",
    "    # Fourth Depthwise Separable Convolution Group\n",
    "    x = group(x, 1024, 2, alpha)\n",
    "    return x\n",
    "    \n",
    "def group(x, n_filters, n_blocks, alpha):\n",
    "    \"\"\" Construct a Depthwise Separable Convolution Group\n",
    "        x         : input to the group\n",
    "        n_filters : number of filters\n",
    "        n_blocks  : number of blocks in the group\n",
    "        alpha     : width multiplier\n",
    "    \"\"\" \n",
    "    # In first block, the depthwise convolution is strided - feature map size reduction\n",
    "    # Replace the ?? with the thinning factor\n",
    "    # HINT: the name of the parameter passed to this function\n",
    "    x = depthwise_block(x, n_filters, ??, strides=(2, 2))\n",
    "    \n",
    "    # Remaining blocks\n",
    "    for _ in range(n_blocks - 1):\n",
    "        x = depthwise_block(x, n_filters, alpha, strides=(1, 1))\n",
    "    return x\n",
    "\n",
    "def depthwise_block(x, n_filters, alpha, strides):\n",
    "    \"\"\" Construct a Depthwise Separable Convolution block\n",
    "        x         : input to the block\n",
    "        n_filters : number of filters\n",
    "        alpha     : width multiplier\n",
    "        strides   : strides\n",
    "    \"\"\"\n",
    "    # Apply the width filter to the number of feature maps\n",
    "    # Replace the ?? with the thinned calculation for the number of filters.\n",
    "    # HINT: multiple the number of filters by the thinning factor. Remember this will be a real number and the convolution layers require\n",
    "    #       an integer, so you will need to cast the result.\n",
    "    filters = ??\n",
    "\n",
    "    # Strided convolution to match number of filters\n",
    "    if strides == (2, 2):\n",
    "        x = ZeroPadding2D(padding=((0, 1), (0, 1)))(x)\n",
    "        padding = 'valid'\n",
    "    else:\n",
    "        padding = 'same'\n",
    "\n",
    "    # Depthwise Convolution\n",
    "    x = DepthwiseConv2D((3, 3), strides, padding=padding)(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = ReLU(6.0)(x)\n",
    "\n",
    "    # Pointwise Convolution\n",
    "    x = Conv2D(filters, (1, 1), strides=(1, 1), padding='same')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = ReLU(6.0)(x)\n",
    "    return x\n",
    "\n",
    "def classifier(x, alpha, dropout, n_classes):\n",
    "    \"\"\" Construct the classifier group\n",
    "        x         : input to the classifier\n",
    "        alpha     : width multiplier\n",
    "        dropout   : dropout percentage\n",
    "        n_classes : number of output classes\n",
    "    \"\"\"\n",
    "    # Flatten the feature maps into 1D feature maps (?, N)\n",
    "    x = GlobalAveragePooling2D()(x)\n",
    "\n",
    "    # Reshape the feature maps to (?, 1, 1, 1024)\n",
    "    shape = (1, 1, int(1024 * alpha))\n",
    "    x = Reshape(shape)(x)\n",
    "    # Perform dropout for preventing overfitting\n",
    "    x = Dropout(dropout)(x)\n",
    "\n",
    "    # Use convolution for classifying (emulates a fully connected layer)\n",
    "    x = Conv2D(n_classes, (1, 1), padding='same', activation='softmax')(x)\n",
    "    # Reshape the resulting output to 1D vector of number of classes\n",
    "    x = Reshape((n_classes, ))(x)\n",
    "    return x\n",
    "\n",
    "# Meta-parameter: width multiplier (0 .. 1) for reducing number of filters.\n",
    "# Replace the ?? with the thinning factor - let's start with no thinning\n",
    "# HINT: one\n",
    "alpha      = ??   \n",
    "\n",
    "# Meta-parameter: dropout rate\n",
    "dropout    = 0.5 \n",
    "\n",
    "inputs = Input(shape=(224, 224, 3))\n",
    "\n",
    "# The Stem Group\n",
    "x = stem(inputs, alpha)    \n",
    "\n",
    "# The Learner\n",
    "x = learner(x, alpha)\n",
    "\n",
    "# The classifier for 1000 classes\n",
    "outputs = classifier(x, alpha, dropout, 1000)\n",
    "\n",
    "# Instantiate the Model\n",
    "model = Model(inputs, outputs)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Verify the model using summary method\n",
    "\n",
    "It end of the output should look like below.\n",
    "\n",
    "```\n",
    "_________________________________________________________________\n",
    "re_lu_26 (ReLU)              (None, 7, 7, 1024)        0         \n",
    "_________________________________________________________________\n",
    "global_average_pooling2d (Gl (None, 1024)              0         \n",
    "_________________________________________________________________\n",
    "reshape (Reshape)            (None, 1, 1, 1024)        0         \n",
    "_________________________________________________________________\n",
    "dropout (Dropout)            (None, 1, 1, 1024)        0         \n",
    "_________________________________________________________________\n",
    "conv2d_14 (Conv2D)           (None, 1, 1, 1000)        1025000   \n",
    "_________________________________________________________________\n",
    "reshape_1 (Reshape)          (None, 1000)              0         \n",
    "=================================================================\n",
    "Total params: 4,264,808\n",
    "Trainable params: 4,242,920\n",
    "Non-trainable params: 21,888\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Try thinning the MobileNet v1\n",
    "\n",
    "Let's now thin this MobileNet v1 you built. Try thinning it by a factor of 0.25. \n",
    "\n",
    "    1. Modify the above code and change alpha = 1, to alpha = 0.25\n",
    "    2. Do a summary() on the model and see how it changed the number of parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Code a Mobile Convolutional Network style Classifier\n",
    "\n",
    "Let's now code the mobile convolutional network style for a classifier, where a convolutional layer is used in place of a dense layer.\n",
    "\n",
    "    1. Set the number of filters in Conv2D to the number of classes.\n",
    "    2. Set the layer to for reducing and flattening the feature maps.\n",
    "    3. Set the final activation function for the class probability distribution.\n",
    "    \n",
    "In the summary, you should see 513K parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras import Input, Model\n",
    "from tensorflow.keras.layers import Conv2D, GlobalAveragePooling2D, Activation\n",
    "\n",
    "def classifier(x, n_classes):\n",
    "    ''' Construct the Classifier \n",
    "        x        : input to the classifier\n",
    "        n_classes: number of output classes\n",
    "    '''\n",
    "    # Repace the ?? with the number of filters\n",
    "    # HINT: set the number of filters equal to number of classes\n",
    "    x = Conv2D(??, (1, 1), strides=1, activation='relu', padding='same')(x)\n",
    "\n",
    "    # reduce each filter (class) to a single value and flatten to a 1D vector\n",
    "    # Replace the ?? with the layer that does global average pooling and flattens into 1D vector\n",
    "    # HINT: the name of the layer is in the the import from tensorflow.keras.layers\n",
    "    x = ??()(x)\n",
    "    \n",
    "    # Replace the ?? with the activation function (string name) used for multi-classification\n",
    "    # HINT: it's the same as if we used a dense layer\n",
    "    outputs = Activation(??)(x)\n",
    "    return outputs\n",
    "\n",
    "# let's pretend this is the final feature map size and numbern before the classifier\n",
    "final_feature_maps = Input((4, 4, 512))\n",
    "model = Model(final_feature_maps, classifier(final_feature_maps, 1000))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Code a SqueezeNet Fire Block\n",
    "\n",
    "Let's code a fire block. You will need to:\n",
    "\n",
    "    1. Set the number of filters for the squeeze layer.\n",
    "    2. Set the inputs to the expand layers.\n",
    "    3. Finish the concatenation of the feature maps from the two expand branches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras import Input, Model\n",
    "from tensorflow.keras.layers import Conv2D, Concatenate\n",
    "\n",
    "def fire_block(x, n_filters):\n",
    "    ''' Construct a Fire Block\n",
    "        x        : input to the block\n",
    "        n_filters: number of filters\n",
    "    '''\n",
    "    # squeeze layer\n",
    "    # Replace the ?? with a bottleneck filter size\n",
    "    # HINT: A bottleneck is a 1 by 1 filter that learns how to reduce size of number of feature maps\n",
    "    squeeze = Conv2D(n_filters, (??, ??), strides=1, activation='relu', padding='same')(x)\n",
    "\n",
    "    # branch the squeeze layer into a 1x1 and 3x3 convolution and double the number of filters\n",
    "    # Replace the ??s with the input from the squeeze layer\n",
    "    # HINT: both convolutional layers have the same input\n",
    "    expand1x1 = Conv2D(n_filters * 4, (1, 1), strides=1, activation='relu', padding='same')(??)\n",
    "    expand3x3 = Conv2D(n_filters * 4, (3, 3), strides=1, activation='relu', padding='same')(??)\n",
    "\n",
    "    # concatenate the feature maps from the 1x1 and 3x3 branches\n",
    "    # Replace the ?? with the output from the 3x3 expand branch\n",
    "    # HINT: it's the branch with the 3x3 filter\n",
    "    x = Concatenate()([expand1x1, ??])\n",
    "    return x\n",
    "\n",
    "# The input shape\n",
    "inputs = Input((224, 224, 3))\n",
    "\n",
    "outputs = fire_block(inputs, 16)\n",
    "\n",
    "# Instantiate the Model\n",
    "model = Model(inputs, outputs)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Verify the block using summary method\n",
    "\n",
    "The output should look like below.\n",
    "```\n",
    "__________________________________________________________________________________________________\n",
    "Layer (type)                    Output Shape         Param #     Connected to                     \n",
    "==================================================================================================\n",
    "input_3 (InputLayer)            [(None, 224, 224, 3) 0                                            \n",
    "__________________________________________________________________________________________________\n",
    "conv2d_3 (Conv2D)               (None, 224, 224, 16) 64          input_3[0][0]                    \n",
    "__________________________________________________________________________________________________\n",
    "conv2d_4 (Conv2D)               (None, 224, 224, 64) 1088        conv2d_3[0][0]                   \n",
    "__________________________________________________________________________________________________\n",
    "conv2d_5 (Conv2D)               (None, 224, 224, 64) 9280        conv2d_3[0][0]                   \n",
    "__________________________________________________________________________________________________\n",
    "concatenate_1 (Concatenate)     (None, 224, 224, 128 0           conv2d_4[0][0]                   \n",
    "                                                                 conv2d_5[0][0]                   \n",
    "==================================================================================================\n",
    "Total params: 10,432\n",
    "Trainable params: 10,432\n",
    "Non-trainable params: 0\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.  Quantize a mobile convolutional network\n",
    "\n",
    "We will start by using a pre-built MobileNetV2 model from Tf.Keras which is trained on ImageNet.\n",
    "\n",
    "Next, we will use the model to do a prediction of the image of an Apple. The integer label returned from the prediction will be 948 (that's an apple)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://github.com/JonathanCMitchell/mobilenet_v2_keras/releases/download/v1.1/mobilenet_v2_weights_tf_dim_ordering_tf_kernels_1.0_224.h5\n",
      "14540800/14536120 [==============================] - 1s 0us/step\n",
      "prediction 948\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.applications import MobileNetV2\n",
    "import cv2\n",
    "import numpy as np\n",
    "\n",
    "# Let's use a prebuilt MobileNet model trained on ImageNet\n",
    "model = MobileNetV2(input_shape=(224, 224, 3), weights='imagenet')\n",
    "\n",
    "# Let's make a prediction with the unquantized (large) version of the model\n",
    "\n",
    "# We will use the image of an apple and preprocess it for the model\n",
    "image = cv2.imread('apple.png')\n",
    "image = cv2.resize(image, (224, 224))\n",
    "image = (image / 255.0).astype(np.float32)\n",
    "\n",
    "# now make the prediction\n",
    "probabilities = model.predict(np.asarray([image]))\n",
    "prediction = np.argmax(probabilities)\n",
    "\n",
    "# Okay, it predicts the label associated with the value 948 (apple)\n",
    "print(\"prediction\", prediction)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quantize the Model\n",
    "\n",
    "Let's now use TFLite to quantatize the model and then do a prediction with the quantatized model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediction 948\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "# Create an instance of the converter for TF.Keras (keras format) model\n",
    "converter = tf.lite.TFLiteConverter.from_keras_model(model)\n",
    "\n",
    "# Convert the model to the TFLite format\n",
    "# Replace the ?? with the method that converts the keras (large) model to a quantatized TFLite model\n",
    "# HINT: method is called convert.\n",
    "tflite_model = converter.??()\n",
    "\n",
    "# Instantiate an interpreter for the TFLite model\n",
    "interpreter = tf.lite.Interpreter(model_content=tflite_model)\n",
    "# Allocate the input and output tensors for the model\n",
    "interpreter.allocate_tensors()\n",
    "\n",
    "# Get input and output tensors details needed for prediction\n",
    "input_details = interpreter.get_input_details()\n",
    "output_details = interpreter.get_output_details()\n",
    "\n",
    "# pass the image as a batch to the input tensor\n",
    "# Replace the ?? with the image of an apple as a batch of 1.\n",
    "# HINT: look back on ho\n",
    "interpreter.set_tensor(input_details[0]['index'], ??)\n",
    "\n",
    "# Execute (invoke) the interpreter to perform the prediction\n",
    "interpreter.invoke()\n",
    "\n",
    "# Get the output from the model\n",
    "softmax = interpreter.get_tensor(output_details[0]['index'])\n",
    "\n",
    "# multi-class example, determine the label predicted from the softmax output\n",
    "prediction = np.argmax(softmax)\n",
    "print(\"prediction\", prediction)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## End of Lab Exercise"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
